{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "26e14d2c",
   "metadata": {},
   "source": [
    "首先是 d2l 里的一些简单东西。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aa79ca42",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# 文本总行数: 3221\n",
      "the time machine by h g wells\n",
      "twinkled and his usually pale face was flushed and animated the\n",
      "['the', 'time', 'machine', 'by', 'h', 'g', 'wells']\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "['i']\n",
      "[]\n",
      "[]\n",
      "['the', 'time', 'traveller', 'for', 'so', 'it', 'will', 'be', 'convenient', 'to', 'speak', 'of', 'him']\n",
      "['was', 'expounding', 'a', 'recondite', 'matter', 'to', 'us', 'his', 'grey', 'eyes', 'shone', 'and']\n",
      "['twinkled', 'and', 'his', 'usually', 'pale', 'face', 'was', 'flushed', 'and', 'animated', 'the']\n"
     ]
    }
   ],
   "source": [
    "import collections\n",
    "import re\n",
    "\n",
    "def read_time_machine():\n",
    "    with open(\"timemachine.txt\", 'r') as f:\n",
    "        lines = f.readlines()\n",
    "    return [re.sub('[^A-Za-z]+', ' ', line).strip().lower() for line in lines]\n",
    "\n",
    "lines = read_time_machine()\n",
    "print(f'# 文本总行数: {len(lines)}')\n",
    "print(lines[0])\n",
    "print(lines[10])\n",
    "\n",
    "def tokenize(lines, token='word'): #@save\n",
    "    \"\"\"将文本行拆分为单词或字符词元\"\"\"\n",
    "    if token == 'word':\n",
    "        return [line.split() for line in lines]\n",
    "    elif token == 'char':\n",
    "        return [list(line) for line in lines]\n",
    "    else:\n",
    "        print('错误：未知词元类型：' + token)\n",
    "\n",
    "tokens = tokenize(lines=lines, token=\"word\")\n",
    "for i in range(11) :\n",
    "    print(tokens[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4eb334e0",
   "metadata": {},
   "source": [
    "实话说，还是调包吧。HF 官方开源了一个 Tokenizers 库，我看就很不错嘛。\n",
    "```cmd\n",
    "pip install tokenizers\n",
    "```\n",
    "似乎 WordPiece 分词器很牛，用一下。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "63be305b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "zathe\n",
      "z\n",
      "##a\n",
      "##t\n",
      "##h\n",
      "##e\n",
      "本次生成的词汇表已保存为 vocab.json\n"
     ]
    }
   ],
   "source": [
    "from tokenizers import BertWordPieceTokenizer\n",
    "import json\n",
    "\n",
    "tokenizer = BertWordPieceTokenizer()\n",
    "tokenizer.train(files=\"timemachine.txt\")\n",
    "print(tokenizer.decode(range(47, 52)))\n",
    "print(tokenizer.id_to_token(47))\n",
    "print(tokenizer.id_to_token(48))\n",
    "print(tokenizer.id_to_token(49))\n",
    "print(tokenizer.id_to_token(50))\n",
    "print(tokenizer.id_to_token(51))\n",
    "\n",
    "# 运行多次可以看到每次的字典都不一样\n",
    "vocab = tokenizer.get_vocab()\n",
    "sorted_vocab = dict(sorted(vocab.items(), key=lambda item: item[1]))\n",
    "\n",
    "with open(\"vocab.json\", \"w\", encoding=\"utf-8\") as f:\n",
    "    json.dump(sorted_vocab, f, ensure_ascii=False, indent=2)\n",
    "\n",
    "print(\"本次生成的词汇表已保存为 vocab.json\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
